# Scrapy settings for shixizeng project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'shixizeng'

SPIDER_MODULES = ['shixizeng.spiders']
NEWSPIDER_MODULE = 'shixizeng.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = ＇sxsandroidapp/4.1.5'

# LOG_LEVEL = "INFO"
# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
# COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
    'X-JSL-API-AUTH': 'md5|1604153188|4761k9Vy|1cf983d4c4ac89b99eee19bcf343a6b5',
    'DeviceToken': 'AqeNhlpRPZEShGLVCC--mpgBrCWn0uchEJ6PyT-4mw7i',
    'utm_source': 'huawei',
    'Host': 'apigateway.shixiseng.com',
    'Cookie': 'SXS_XSESSION_ID=2|1:0|10:1604151638|15:SXS_XSESSION_ID|48:ZGRkOWY1OGItZDY2OC00ZDk5LWI2ZmEtMTY0YjEwYmVmNTNi|13c904900eb551ffc4614446e61f0643de7685a10d9aef31ce1be45c8145da62; SXS_VISIT_XSESSION_ID_V3.0="2|1:0|10:1604151685|26:SXS_VISIT_XSESSION_ID_V3.0|48:NzM5OTY5ZjktOTM0Zi00NmYxLTljODItOTg4MGQwMTgyY2U2|a0497e7860f347d26b3f14e0cf9a4b93af3dfcef54ffcb7a02fcc1b16807cf59"; SXS_VISIT_XSESSION_ID_V3.0_EXP="2|1:0|10:1604151685|30:SXS_VISIT_XSESSION_ID_V3.0_EXP|16:MTYwNjc0MzY4NQ==|49d3f2a5a8b937e587ee7fd3c0c6f629a55898d3054ed3ec8ca80b10fa29bacb"; __jsluid_s=8896107bcf24f6ab31e8174de237ef15; affefdgx=usr_yeqkvlwitcgb; sxs_usr=2|1:0|10:1604151638|7:sxs_usr|24:dXNyX3llcWt2bHdpdGNnYg==|de3542a2658f4aa546dc4906304468f4982b3f6bb7644bed045696e270cdaf07; SXS_XSESSION_ID_EXP=2|1:0|10:1604151638|19:SXS_XSESSION_ID_EXP|16:MTYwNjc0MzYzOA==|80dc5795a5282b46755b48c49ba474b61cb2e34fc15ad3805ab2117c8dbcc7e0; xyz_usr=2|1:0|10:1604151638|7:xyz_usr|40:b0MwMG52M1h2SnE3bmdZV3lIdWVKd3g2LTNiQQ==|2b97e1be95abf78acf134ab63e837c32ae10d8241cae84488c96d1ab6d57ff91',

}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
SPIDER_MIDDLEWARES = {
    'shixizeng.middlewares.ShixizengSpiderMiddleware': 543,
}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
    'shixizeng.middlewares.ShixizengDownloaderMiddleware': 543,
}

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
# EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
# }

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'shixizeng.pipelines.ShixizengPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
# AUTOTHROTTLE_ENABLED = True
# The initial download delay
# AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
# AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
# AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
# HTTPCACHE_ENABLED = True
# HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
